result = query_api.query_data_frame(org=org, query=query)
result.columnsIndex(['result', 'table', '_start', '_stop', '_time', '_value', '_field',
'_measurement', 'device', 'sensor', 'units'],
dtype='object')
1 hour integral of power in kWh
2021/04/01 - 2023/11/09
Index(['result', 'table', '_start', '_stop', '_time', '_value', '_field',
'_measurement', 'device', 'sensor', 'units'],
dtype='object')
result.to_csv(f’./Data/energy_info.csv’, index=False) #save the data as a csv
| _time | _value | sensor | |
|---|---|---|---|
| 0 | 2023-03-02 00:00:00+00:00 | 12.888000 | Aircon |
| 1 | 2023-03-03 00:00:00+00:00 | 121.541007 | Aircon |
| 2 | 2023-03-04 00:00:00+00:00 | 222.993806 | Aircon |
| 3 | 2023-03-05 00:00:00+00:00 | 118.435757 | Aircon |
| 4 | 2023-03-06 00:00:00+00:00 | 238.721389 | Aircon |
wide_df = result.pivot(index='_time', columns='sensor', values='_value')
# Reset the index to make 'id' a regular column
wide_df.reset_index(inplace=True)
wide_df.columnsIndex(['_time', 'Aircon', 'Consumption', 'ConsumptionNet', 'Export', 'Fridge',
'Garage', 'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR',
'HouseBoardR', 'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints',
'Production', 'Unmonitored'],
dtype='object', name='sensor')
wide_df_new = pd.DataFrame()
for column in wide_df.columns:
wide_df_new[f'{column}'] = wide_df[f'{column}'].interpolate()
wide_df = pd.DataFrame(wide_df_new)
wide_df.columnsIndex(['ds', 'Aircon', 'Consumption', 'ConsumptionNet', 'Export', 'Fridge',
'Garage', 'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR',
'HouseBoardR', 'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints',
'Production'],
dtype='object')
| ds | Aircon | Consumption | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | HouseBoardR | Import | Lights | Net | OvenStove | Powerpoints | Production | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 240 | 2023-10-28 | 8.578005 | 357.275503 | 866.654129 | 180.242247 | 25.492343 | 24.017087 | 0.530865 | 999.775552 | 333.258545 | 366.408449 | 71.876417 | 80.009421 | -108.365830 | 66.086281 | 176.392548 | 491.748285 |
| 241 | 2023-10-29 | 8.661682 | 179.501779 | 866.654129 | 806.610093 | 19.634587 | 2.731102 | 0.192283 | 530.312474 | 176.770805 | 201.397413 | 1.974071 | 60.108551 | -804.636021 | 1.253300 | 103.451046 | 1017.375874 |
| 242 | 2023-10-30 | 8.603354 | 263.812291 | 866.654129 | 757.601472 | 27.081000 | 0.983716 | 0.183639 | 788.486015 | 262.828480 | 290.616882 | 3.378339 | 66.827993 | -754.223132 | 1.356785 | 178.170521 | 1052.272015 |
| 243 | 2023-10-31 | 9.032917 | 311.552394 | 866.654129 | 803.841970 | 35.649472 | 11.163953 | 0.269313 | 901.165305 | 300.388379 | 319.795146 | 5.027126 | 67.972347 | -798.814844 | 30.217549 | 166.649785 | 1142.661342 |
| 244 | 2023-11-01 | 8.743938 | 211.339011 | 866.654129 | 898.594991 | 30.389375 | 3.655479 | 0.189410 | 623.050504 | 207.683462 | 245.486118 | 2.309883 | 50.624125 | -896.285107 | 0.213375 | 145.941312 | 1134.978184 |
def long_form(df_plot):
return df_plot.melt('ds', var_name='unique_id', value_name='y')
def altair_plot(df_plot):
import altair as alt
highlight = alt.selection_point(on='mouseover', fields=['unique_id'], bind='legend', nearest=True)
base = alt.Chart(df_plot).encode(
x='ds:T',
y='y:Q',
color='unique_id:N'
)
points = base.mark_circle().encode(
opacity=alt.value(0)
).add_params(
highlight
).properties(
width=1000
)
lines = base.mark_line().encode(
size=alt.condition(~highlight, alt.value(1), alt.value(3))
)
return points + lines| ds | Aircon | Consumption | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | HouseBoardR | Import | Lights | Net | OvenStove | Powerpoints | Production | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 240 | 2023-10-28 | 0.000327 | 0.295413 | 0.639383 | 0.087269 | 0.054518 | 0.475787 | 0.000988 | 0.441317 | 0.270647 | 0.263152 | 0.098826 | 0.397064 | 0.702175 | 0.217335 | 0.451213 | 0.164710 |
| 241 | 2023-10-29 | 0.000459 | 0.096052 | 0.639383 | 0.390543 | 0.008991 | 0.049113 | 0.000167 | 0.155476 | 0.095349 | 0.096613 | 0.001842 | 0.287229 | 0.452268 | 0.003559 | 0.195780 | 0.385757 |
| 242 | 2023-10-30 | 0.000367 | 0.190600 | 0.639383 | 0.366814 | 0.066865 | 0.014087 | 0.000146 | 0.312670 | 0.191751 | 0.186659 | 0.003790 | 0.324314 | 0.470363 | 0.003900 | 0.457440 | 0.400433 |
| 243 | 2023-10-31 | 0.001045 | 0.244137 | 0.639383 | 0.389203 | 0.133461 | 0.218148 | 0.000354 | 0.381277 | 0.233826 | 0.216107 | 0.006078 | 0.330630 | 0.454358 | 0.099064 | 0.417095 | 0.438445 |
| 244 | 2023-11-01 | 0.000589 | 0.131755 | 0.639383 | 0.435080 | 0.092579 | 0.067642 | 0.000160 | 0.211942 | 0.129978 | 0.141110 | 0.002308 | 0.234883 | 0.419373 | 0.000130 | 0.344577 | 0.435214 |
import pandas as pd
path = 'Data'
name = 'weather'
weather_df = pd.read_csv(f'{path}/{name}.csv')
weather_df.head()| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-01-01 00:00 | 21.6 | 0 | 12.6 | 126 | 1013.0 | 0.0 | 84 | 73 | 21.6 | 21.6 | 24.4 | 18.7 | 0 | 10.0 | 19.8 | 1.0 |
| 1 | 2023-01-01 01:00 | 21.3 | 0 | 11.9 | 133 | 1013.0 | 0.0 | 85 | 77 | 21.3 | 21.3 | 24.3 | 18.7 | 0 | 10.0 | 18.7 | 1.0 |
| 2 | 2023-01-01 02:00 | 21.1 | 0 | 11.2 | 140 | 1012.0 | 0.0 | 86 | 80 | 21.1 | 21.1 | 24.2 | 18.7 | 0 | 10.0 | 17.6 | 1.0 |
| 3 | 2023-01-01 03:00 | 20.8 | 0 | 10.4 | 147 | 1012.0 | 0.0 | 88 | 84 | 20.8 | 20.8 | 20.8 | 18.7 | 0 | 10.0 | 16.6 | 1.0 |
| 4 | 2023-01-01 04:00 | 21.0 | 0 | 10.6 | 144 | 1012.0 | 0.0 | 86 | 84 | 21.0 | 21.0 | 21.0 | 18.6 | 0 | 10.0 | 16.3 | 1.0 |
def data_day_avg(weather_df):
from tqdm.notebook import tqdm
avg_df = pd.DataFrame()
dates = pd.to_datetime(weather_df['ds']).dt.date.unique()
for date in tqdm(dates):
filtered_df = weather_df[pd.to_datetime(weather_df['ds']).dt.date == date]
ds = filtered_df.pop('ds')
filtered_df = pd.DataFrame(filtered_df.mean()).T
filtered_df
filtered_df.insert(0, 'ds', date)
avg_df = pd.concat([avg_df, filtered_df], ignore_index=True)
return avg_df| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-01-01 | 0.566503 | 0.583333 | 0.459098 | 0.345636 | 0.428728 | 0.000895 | 0.670886 | 0.583750 | 0.516791 | 0.595085 | 0.512881 | 0.778121 | 0.0 | 1.000000 | 0.434678 | 0.422619 |
| 1 | 2023-01-02 | 0.554902 | 0.583333 | 0.451453 | 0.396936 | 0.359649 | 0.001253 | 0.602848 | 0.409167 | 0.499005 | 0.584249 | 0.486067 | 0.725707 | 0.0 | 1.000000 | 0.434844 | 0.428571 |
| 2 | 2023-01-03 | 0.582516 | 0.583333 | 0.263889 | 0.415506 | 0.258772 | 0.000000 | 0.584916 | 0.287917 | 0.525746 | 0.610043 | 0.502366 | 0.738663 | 0.0 | 1.000000 | 0.283533 | 0.452381 |
| 3 | 2023-01-04 | 0.650490 | 0.583333 | 0.313073 | 0.382660 | 0.163377 | 0.000895 | 0.687764 | 0.196250 | 0.608582 | 0.673535 | 0.609621 | 0.866902 | 0.0 | 0.994792 | 0.305777 | 0.500000 |
| 4 | 2023-01-05 | 0.654575 | 0.583333 | 0.257518 | 0.506267 | 0.166667 | 0.026859 | 0.735232 | 0.700417 | 0.629726 | 0.677350 | 0.608701 | 0.911808 | 0.0 | 0.989583 | 0.288098 | 0.416667 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 339 | 2023-12-06 | 0.580556 | 0.583333 | 0.254077 | 0.264740 | 0.558114 | 0.000000 | 0.608650 | 0.174583 | 0.519527 | 0.608364 | 0.515116 | 0.745878 | 0.0 | 1.000000 | 0.214558 | 0.470238 |
| 340 | 2023-12-07 | 0.605065 | 0.583333 | 0.266565 | 0.202066 | 0.565789 | 0.000000 | 0.582806 | 0.050000 | 0.543532 | 0.631258 | 0.533386 | 0.748233 | 0.0 | 1.000000 | 0.222693 | 0.476190 |
| 341 | 2023-12-08 | 0.618954 | 0.583333 | 0.349261 | 0.251625 | 0.575658 | 0.000018 | 0.626582 | 0.342083 | 0.561816 | 0.644078 | 0.555205 | 0.795200 | 0.0 | 1.000000 | 0.285940 | 0.476190 |
| 342 | 2023-12-09 | 0.612582 | 0.583333 | 0.362768 | 0.190460 | 0.544956 | 0.000107 | 0.577532 | 0.497500 | 0.564179 | 0.638584 | 0.539432 | 0.757509 | 0.0 | 1.000000 | 0.308267 | 0.428571 |
| 343 | 2023-12-10 | 0.630556 | 0.583333 | 0.513507 | 0.233751 | 0.530702 | 0.000233 | 0.515295 | 0.537500 | 0.571269 | 0.655067 | 0.546924 | 0.730271 | 0.0 | 1.000000 | 0.426627 | 0.452381 |
344 rows × 17 columns
/home/ben/mambaforge/envs/cfast/lib/python3.11/site-packages/statsforecast/core.py:25: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)
from tqdm.autonotebook import tqdm
from pylab import rcParams
rcParams['figure.figsize'] = (15,4)
StatsForecast.plot(df, engine='plotly')/home/ben/mambaforge/envs/cfast/lib/python3.11/site-packages/_plotly_utils/basevalidators.py:105: FutureWarning:
The behavior of DatetimeProperties.to_pydatetime is deprecated, in a future version this will return a Series containing python datetime objects instead of an ndarray. To retain the old behavior, call `np.array` on the result
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 245 entries, 0 to 244
Data columns (total 18 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 245 non-null datetime64[ns]
1 Aircon 245 non-null float64
2 y 245 non-null float64
3 ConsumptionNet 198 non-null float64
4 Export 245 non-null float64
5 Fridge 245 non-null float64
6 Garage 245 non-null float64
7 Hotwater 245 non-null float64
8 HouseBoardOutside 161 non-null float64
9 HouseBoardOutsideR 245 non-null float64
10 HouseBoardR 245 non-null float64
11 Import 245 non-null float64
12 Lights 245 non-null float64
13 Net 245 non-null float64
14 OvenStove 245 non-null float64
15 Powerpoints 245 non-null float64
16 Production 245 non-null float64
17 unique_id 245 non-null object
dtypes: datetime64[ns](1), float64(16), object(1)
memory usage: 34.6+ KB
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3920 entries, 0 to 3919
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 3920 non-null datetime64[ns]
1 unique_id 3920 non-null object
2 y 3789 non-null float64
dtypes: datetime64[ns](1), float64(1), object(1)
memory usage: 92.0+ KB
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 344 entries, 0 to 343
Data columns (total 17 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 344 non-null datetime64[ns]
1 temp_c 344 non-null float64
2 is_day 344 non-null float64
3 wind_kph 344 non-null float64
4 wind_degree 344 non-null float64
5 pressure_mb 344 non-null float64
6 precip_mm 344 non-null float64
7 humidity 344 non-null float64
8 cloud 344 non-null float64
9 feelslike_c 344 non-null float64
10 windchill_c 344 non-null float64
11 heatindex_c 344 non-null float64
12 dewpoint_c 344 non-null float64
13 chance_of_rain 344 non-null float64
14 vis_km 344 non-null float64
15 gust_kph 344 non-null float64
16 uv 344 non-null float64
dtypes: datetime64[ns](1), float64(16)
memory usage: 45.8 KB
date_range_start, date_range_end = wide_df['ds'].min(), wide_df['ds'].max()
# Filter rows between the start and end dates
weather_avg_filtered_df = weather_avg_df[(weather_avg_df['ds'] >= date_range_start) & (weather_avg_df['ds'] <= date_range_end)]
weather_avg_filtered_df| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 60 | 2023-03-02 | 0.644444 | 0.541667 | 0.294980 | 0.153784 | 0.344298 | 0.001056 | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 61 | 2023-03-03 | 0.603268 | 0.541667 | 0.483563 | 0.386142 | 0.403509 | 0.002131 | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 62 | 2023-03-04 | 0.561765 | 0.541667 | 0.574669 | 0.433612 | 0.425439 | 0.000161 | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 63 | 2023-03-05 | 0.598039 | 0.541667 | 0.372324 | 0.350627 | 0.427632 | 0.000358 | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 64 | 2023-03-06 | 0.629575 | 0.541667 | 0.313073 | 0.290854 | 0.417763 | 0.000358 | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 300 | 2023-10-28 | 0.379575 | 0.541667 | 0.700051 | 0.446611 | 0.708333 | 0.000233 | 0.500000 | 0.389167 | 0.351741 | 0.414835 | 0.327681 | 0.497939 | 0.0 | 1.0 | 0.588811 | 0.345238 |
| 301 | 2023-10-29 | 0.382516 | 0.541667 | 0.383028 | 0.320682 | 0.656798 | 0.000125 | 0.524789 | 0.433750 | 0.347637 | 0.423230 | 0.329390 | 0.525324 | 0.0 | 1.0 | 0.328934 | 0.345238 |
| 302 | 2023-10-30 | 0.441340 | 0.583333 | 0.396152 | 0.435237 | 0.562500 | 0.000000 | 0.583861 | 0.181250 | 0.403483 | 0.478327 | 0.373686 | 0.611307 | 0.0 | 1.0 | 0.340139 | 0.416667 |
| 303 | 2023-10-31 | 0.550163 | 0.583333 | 0.489424 | 0.549559 | 0.442982 | 0.000000 | 0.570675 | 0.025417 | 0.492537 | 0.579976 | 0.467140 | 0.687132 | 0.0 | 1.0 | 0.470286 | 0.464286 |
| 304 | 2023-11-01 | 0.510784 | 0.583333 | 0.561672 | 0.445218 | 0.505482 | 0.000340 | 0.599156 | 0.480000 | 0.458209 | 0.543040 | 0.435594 | 0.685218 | 0.0 | 1.0 | 0.467795 | 0.404762 |
245 rows × 17 columns
0 0.355157
1 0.291076
2 0.498418
3 0.418824
4 0.561599
...
240 0.295413
241 0.096052
242 0.190600
243 0.244137
244 0.131755
Name: y, Length: 245, dtype: float64
fig, axs = plt.subplots(nrows=1, ncols=2)
plot_acf(wide_df["y"], lags=28, ax=axs[0],color="fuchsia")
axs[0].set_title("Autocorrelation");
plot_pacf(wide_df["y"], lags=28, ax=axs[1],color="lime")
axs[1].set_title('Partial Autocorrelation')
plt.show();
| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 60 | 2023-03-02 | 0.644444 | 0.541667 | 0.294980 | 0.153784 | 0.344298 | 0.001056 | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 61 | 2023-03-03 | 0.603268 | 0.541667 | 0.483563 | 0.386142 | 0.403509 | 0.002131 | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 62 | 2023-03-04 | 0.561765 | 0.541667 | 0.574669 | 0.433612 | 0.425439 | 0.000161 | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 63 | 2023-03-05 | 0.598039 | 0.541667 | 0.372324 | 0.350627 | 0.427632 | 0.000358 | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 64 | 2023-03-06 | 0.629575 | 0.541667 | 0.313073 | 0.290854 | 0.417763 | 0.000358 | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
| ds | Aircon | y | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | ... | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-03-02 | 0.007134 | 0.355157 | NaN | 0.016595 | 0.814609 | 0.003794 | 0.000287 | NaN | 0.356702 | ... | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 1 | 2023-03-03 | 0.178748 | 0.291076 | NaN | 0.165011 | 0.669709 | 0.015031 | 0.000143 | NaN | 0.292065 | ... | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 2 | 2023-03-04 | 0.338990 | 0.498418 | NaN | 0.294620 | 0.686509 | 0.059559 | 0.000460 | NaN | 0.496691 | ... | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 3 | 2023-03-05 | 0.173844 | 0.418824 | NaN | 0.302607 | 0.701288 | 0.009916 | 0.000256 | NaN | 0.419958 | ... | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 4 | 2023-03-06 | 0.363831 | 0.561599 | NaN | 0.381220 | 0.773714 | 0.008798 | 0.002112 | NaN | 0.562640 | ... | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
5 rows × 34 columns
horizon = 28 * 2
train_size = len(wide_df) - horizon
train, test = wide_df[:train_size], wide_df[train_size:]
wide_df.shape, train.shape, test.shape((245, 34), (189, 34), (56, 34))
/tmp/ipykernel_1270693/3311167277.py:2: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
(56, 18)
| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | unique_id | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 249 | 2023-09-07 | 0.430882 | 0.5 | 0.251784 | 0.298630 | 0.699561 | 0.000000 | 0.748418 | 0.322083 | 0.397637 | 0.468407 | 0.371320 | 0.703475 | 0.0 | 1.000000 | 0.223274 | 0.345238 | Consumption |
| 250 | 2023-09-08 | 0.433660 | 0.5 | 0.348114 | 0.424559 | 0.656798 | 0.001612 | 0.732595 | 0.320417 | 0.394776 | 0.471001 | 0.371845 | 0.694052 | 0.0 | 0.989583 | 0.321547 | 0.345238 | Consumption |
| 251 | 2023-09-09 | 0.338889 | 0.5 | 0.389016 | 0.465761 | 0.732456 | 0.000358 | 0.443038 | 0.115833 | 0.308706 | 0.378816 | 0.283123 | 0.415931 | 0.0 | 1.000000 | 0.370020 | 0.309524 | Consumption |
| 252 | 2023-09-10 | 0.292810 | 0.5 | 0.490571 | 0.406685 | 0.789474 | 0.000000 | 0.489979 | 0.264167 | 0.268159 | 0.329060 | 0.235410 | 0.428445 | 0.0 | 1.000000 | 0.432852 | 0.267857 | Consumption |
| 253 | 2023-09-11 | 0.326471 | 0.5 | 0.583716 | 0.429085 | 0.822368 | 0.000000 | 0.456751 | 0.294583 | 0.298881 | 0.366758 | 0.262487 | 0.430654 | 0.0 | 1.000000 | 0.511288 | 0.303571 | Consumption |
['RNN',
'GRU',
'LSTM',
'TCN',
'DeepAR',
'DilatedRNN',
'MLP',
'NHITS',
'NBEATS',
'NBEATSx',
'TFT',
'VanillaTransformer',
'Informer',
'Autoformer',
'PatchTST',
'FEDformer',
'StemGNN',
'HINT',
'TimesNet']
model_list = [RNN,
GRU,
LSTM,
TCN,
#DeepAR, # not good
DilatedRNN,
MLP,
NHITS,
NBEATS,
NBEATSx,
#TFT, # too much GPU
#VanillaTransformer, # not good
#Informer,
#Autoformer,
#PatchTST, # too much GPU
#FEDformer, #taken too long
#StemGNN, #need n_series
#HINT, #need n_series
#TimesNet # takes too long
]Index(['ds', 'Aircon', 'y', 'ConsumptionNet', 'Export', 'Fridge', 'Garage',
'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR', 'HouseBoardR',
'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints', 'Production',
'unique_id', 'temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain', 'vis_km',
'gust_kph', 'uv'],
dtype='object')
models = [model(input_size=2 * horizon,
h=horizon,
max_steps=50,
hist_exog_list =['Aircon', 'Export', 'Fridge', 'Garage',
'Hotwater',
'HouseBoardR','Import', 'Lights', 'Net', 'OvenStove',
'Powerpoints', 'Production'
],
futr_exog_list = ['temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain',
'vis_km', 'gust_kph', 'uv']
) for model in model_list]
modelsGlobal seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
[RNN, GRU, LSTM, TCN, DilatedRNN, MLP, NHITS, NBEATS, NBEATSx]
Index(['ds', 'Aircon', 'y', 'ConsumptionNet', 'Export', 'Fridge', 'Garage',
'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR', 'HouseBoardR',
'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints', 'Production',
'unique_id', 'temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain', 'vis_km',
'gust_kph', 'uv'],
dtype='object')
| unique_id | ds | RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | Consumption | 2023-09-07 | 0.520451 | 0.558734 | 0.566012 | 0.552208 | 0.565061 | 0.398763 | 0.235965 | 0.266845 | 0.296138 |
| 1 | Consumption | 2023-09-08 | 0.493308 | 0.514300 | 0.395049 | 0.494308 | 0.495290 | 0.387929 | 0.224346 | 0.191190 | 0.212631 |
| 2 | Consumption | 2023-09-09 | 0.425334 | 0.355697 | 0.433629 | 0.400758 | 0.381633 | 0.408256 | 0.241371 | 0.233107 | 0.228564 |
| 3 | Consumption | 2023-09-10 | 0.443577 | 0.498908 | 0.442869 | 0.366124 | 0.378713 | 0.407515 | 0.233445 | 0.234206 | 0.226971 |
| 4 | Consumption | 2023-09-11 | 0.353572 | 0.434181 | 0.394170 | 0.391079 | 0.314454 | 0.401142 | 0.212357 | 0.289439 | 0.254854 |
consumption_predict_plot = long_form(consumption_predict)
consumption_predict_plot = pd.concat([consumption_predict_plot, just_consumption]
, ignore_index=True)
consumption_predict_plot| ds | unique_id | y | |
|---|---|---|---|
| 0 | 2023-09-07 | RNN | 0.520451 |
| 1 | 2023-09-08 | RNN | 0.493308 |
| 2 | 2023-09-09 | RNN | 0.425334 |
| 3 | 2023-09-10 | RNN | 0.443577 |
| 4 | 2023-09-11 | RNN | 0.353572 |
| ... | ... | ... | ... |
| 744 | 2023-10-28 | Consumption | 0.295413 |
| 745 | 2023-10-29 | Consumption | 0.096052 |
| 746 | 2023-10-30 | Consumption | 0.190600 |
| 747 | 2023-10-31 | Consumption | 0.244137 |
| 748 | 2023-11-01 | Consumption | 0.131755 |
749 rows × 3 columns
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 56 entries, 0 to 55
Data columns (total 10 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 56 non-null datetime64[ns]
1 RNN 56 non-null float64
2 GRU 56 non-null float64
3 LSTM 56 non-null float64
4 TCN 56 non-null float64
5 DilatedRNN 56 non-null float64
6 MLP 56 non-null float64
7 NHITS 56 non-null float64
8 NBEATS 56 non-null float64
9 NBEATSx 56 non-null float64
dtypes: datetime64[ns](1), float64(9)
memory usage: 4.5 KB
(None,
Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object'))
def evaluate_performace(y_hist, y_true, y_pred):
#y_true = y_true.merge(y_pred, how='left', on=['unique_id', 'ds'])
evaluation = {}
print(y_pred.columns)
for i, model in enumerate(y_pred.columns):
if i < 1: continue
evaluation[model] = {}
for metric in [mase, mae, mape, rmse, smape]:
metric_name = metric.__name__
if metric_name == 'mase':
evaluation[model][metric_name] = metric(y_true['y'].values,
y_pred[model].values,
y_hist['y'].values, seasonality=12)
else:
evaluation[model][metric_name] = metric(y_true['y'].values, y_pred[model].values)
return pd.DataFrame(evaluation)score = evaluate_performace(train, test, consumption_predict)
score.style.background_gradient(cmap ='YlGn', axis = 1,low=0.5, high=0.6)Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object')
| RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|
| mase | 0.708069 | 0.753612 | 0.653206 | 0.715462 | 0.683945 | 0.841424 | 0.585120 | 0.644987 | 0.540987 |
| mae | 0.116573 | 0.124071 | 0.107540 | 0.117790 | 0.112601 | 0.138528 | 0.096331 | 0.106187 | 0.089065 |
| mape | 117.972952 | 123.586528 | 92.569209 | 117.289240 | 113.036232 | 141.320639 | 63.815475 | 109.551444 | 72.312216 |
| rmse | 0.144370 | 0.158492 | 0.138631 | 0.148289 | 0.142224 | 0.161854 | 0.127389 | 0.136658 | 0.117224 |
| smape | 53.985201 | 55.843691 | 52.182055 | 53.097440 | 52.532795 | 59.642131 | 62.462127 | 55.998253 | 53.806687 |
Index(['unique_id', 'ds', 'cutoff', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN',
'MLP', 'NHITS', 'NBEATS', 'NBEATSx', 'Aircon', 'y', 'ConsumptionNet',
'Export', 'Fridge', 'Garage', 'Hotwater', 'HouseBoardOutside',
'HouseBoardOutsideR', 'HouseBoardR', 'Import', 'Lights', 'Net',
'OvenStove', 'Powerpoints', 'Production', 'temp_c', 'is_day',
'wind_kph', 'wind_degree', 'pressure_mb', 'precip_mm', 'humidity',
'cloud', 'feelslike_c', 'windchill_c', 'heatindex_c', 'dewpoint_c',
'chance_of_rain', 'vis_km', 'gust_kph', 'uv'],
dtype='object')
0 0.435481
1 0.571911
2 0.712023
3 0.549668
4 0.572973
...
163 0.331751
164 0.253077
165 0.284961
166 0.376310
167 0.242600
Name: RNN, Length: 168, dtype: float32
Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object')
| ds | RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-07-11 | 0.435481 | 0.441562 | 0.498433 | 0.483206 | 0.500122 | 0.580595 | 0.490572 | 0.561005 | 0.524275 |
| 1 | 2023-07-12 | 0.571911 | 0.569870 | 0.579989 | 0.628650 | 0.598320 | 0.679515 | 0.452369 | 0.516658 | 0.289955 |
| 2 | 2023-07-13 | 0.712023 | 0.640818 | 0.719223 | 0.676610 | 0.605447 | 0.670375 | 0.338826 | 0.413797 | 0.194005 |
| 3 | 2023-07-14 | 0.549668 | 0.612540 | 0.583302 | 0.630741 | 0.591179 | 0.687667 | 0.341010 | 0.349947 | 0.284295 |
| 4 | 2023-07-15 | 0.572973 | 0.515143 | 0.499026 | 0.550076 | 0.550768 | 0.460477 | 0.225505 | 0.478275 | 0.333824 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 163 | 2023-09-02 | 0.331751 | 0.273870 | 0.252772 | 0.315278 | 0.250863 | 0.163495 | 0.206029 | 0.343418 | 0.262445 |
| 164 | 2023-09-03 | 0.253077 | 0.223345 | 0.258903 | 0.295469 | 0.289489 | 0.261679 | 0.284491 | 0.301202 | 0.263286 |
| 165 | 2023-09-04 | 0.284961 | 0.200487 | 0.221326 | 0.324679 | 0.294298 | 0.254948 | 0.422089 | 0.328173 | 0.222025 |
| 166 | 2023-09-05 | 0.376310 | 0.339444 | 0.321259 | 0.406183 | 0.381957 | 0.062036 | 0.235567 | 0.484579 | -0.211019 |
| 167 | 2023-09-06 | 0.242600 | 0.215008 | 0.229029 | 0.244884 | 0.185244 | 0.396192 | 0.361900 | 0.446674 | 0.201929 |
168 rows × 10 columns
| ds | unique_id | y | |
|---|---|---|---|
| 0 | 2023-07-11 | RNN | 0.435481 |
| 1 | 2023-07-12 | RNN | 0.571911 |
| 2 | 2023-07-13 | RNN | 0.712023 |
| 3 | 2023-07-14 | RNN | 0.549668 |
| 4 | 2023-07-15 | RNN | 0.572973 |
| ... | ... | ... | ... |
| 1507 | 2023-09-02 | NBEATSx | 0.262445 |
| 1508 | 2023-09-03 | NBEATSx | 0.263286 |
| 1509 | 2023-09-04 | NBEATSx | 0.222025 |
| 1510 | 2023-09-05 | NBEATSx | -0.211019 |
| 1511 | 2023-09-06 | NBEATSx | 0.201929 |
1512 rows × 3 columns